import asyncio

from com.arcfox.base.base_spider import BaseSpider, ExecuteType
from com.arcfox.middleware.async_redis_middleware import open_redis
from com.arcfox.school.processor.rank.ruanke_school_processor import RuanKeSchoolProcessor
from com.arcfox.util import async_request as requests

'''
    软科排名数据抓取
'''


class RuanKeSpider(BaseSpider):

    def __init__(self):
        super().__init__()
        self.urls = [
            # 学校排名
            "https://www.shanghairanking.cn/_nuxt/static/1650334172/rankings/bcur/202211/payload.js",
            "https://www.shanghairanking.cn/_nuxt/static/1650334172/rankings/bcur/202110/payload.js",
            "https://www.shanghairanking.cn/_nuxt/static/1650334172/rankings/bcur/202011/payload.js",
        ]
        self.processor = RuanKeSchoolProcessor()

    async def _pull_task(self):
        return self.urls, ExecuteType.ONCE

    async def _crawl_by_task(self, tasks):
        for url in tasks:
            resp = await requests.get_with_session(url, headers=self.get_default_header())
            if resp.code == 200:
                await self.processor.parse_and_save_data(resp.response)

    @open_redis
    async def init_data_version(self, client):
        pass


if __name__ == "__main__":
    asyncio.run(RuanKeSpider().start_crawl())
